#encoding: utf-8
# -*- coding: utf-8 -*-
import codecs
import os,sys
 
os.chdir('E:\\desktop\\wu-request\\NLPCC 2014 Shared Tasks Guidelines\\crfNER')
if True:
    fd = codecs.open('people-daily-bamboo-edition.txt','r',encoding='utf-8') 
    raw = fd.read()
    fd.close()
    lines=raw.split('\n')
    
    corpor=[]
    for ln in lines:
        tmp_words=ln.split(' ')
        words = [itm for itm in tmp_words if len(itm.strip())>0 ]
        corpor.append(words[1:])
    
    
    clear_corpor = []
    for line in corpor:
        new_line=[]
        for word in line:
            word=word.strip()
            if len(word)>0 and word[0]=='[':
                word=word[1:]
            if len(word)>3 and word[-3]==']':
                word=word[0:-3]
            if len(word)>2 and word[-2]==']':
                word=word[0:-2]
            new_line.append(word)
        clear_corpor.append(new_line)
    
    spt_idx = int(len(clear_corpor)*0.9)
    train_corpus = clear_corpor[0:spt_idx]
    test_corpus = clear_corpor[spt_idx+1:]

'建立POS和SEG的训练和测试数据'
if False: 
    fd = codecs.open('people_daily_train.txt','w',encoding='gbk') 
    for ln in train_corpus:
        for wd in ln:
            ws = wd.split(u'/')
            assert(len(ws)==2 or ws=='')
            fd.write(ws[0]+' '+ws[1]+' '+ws[1]+'\n')
        fd.write('\n')
    fd.close()
    
    fd = codecs.open('people_daily_train_seg.txt','w',encoding='gbk') 
    for ln in train_corpus:
        for wd in ln:
            ws = wd.split(u'/')
            assert(len(ws)==2)
            word = ws[0]
            for idx in range(0,len(word)):
                if idx==0:
                    fd.write(word[idx]+' B B\n')
                elif idx==len(word)-1:
                    fd.write(word[idx]+' E E\n')
                else:
                    fd.write(word[idx]+' I I\n')
        fd.write('\n')
    fd.close()
    
    fd = codecs.open('people_daily_test.txt','w',encoding='gbk') 
    for ln in test_corpus:
        for wd in ln:
            ws = wd.split(u'/')
            fd.write(ws[0]+' '+ws[1]+'\n')
        fd.write('\n')
    fd.close()
    
    fd = codecs.open('people_daily_test_seg.txt','w',encoding='gbk') 
    for ln in test_corpus:
        for wd in ln:
            ws = wd.split(u'/')
            assert(len(ws)==2)
            word = ws[0]
            for idx in range(0,len(word)):
                if idx==0:
                    fd.write(word[idx]+' B B\n')
                elif idx==len(word)-1:
                    fd.write(word[idx]+' E E\n')
                else:
                    fd.write(word[idx]+' I I\n')
        fd.write('\n')
    fd.close()
   
'建立NAMED ENTITY的训练和测试数据'
if True:
    neclear_corpor = []
    for line in corpor:
        new_line=[]
        state_idx = 'O'
        last_tag = ''
        for word in line:
            word=word.strip()
            if len(word)>0 and word[0]=='[':
                word=word[1:]
                last_tag=''
                state_idx = 'B'
            if len(word)>3 and word[-3]==']':
                state_idx = 'E'
                last_tag = word[-2:]
                word=word[0:-3]
            if len(word)>2 and word[-2]==']':
                state_idx = 'E'
                last_tag = word[-1:]
                word=word[0:-2]
            
            words = word.split(u'/')
            assert(len(words)==2)
            if last_tag=='':
                new_line.append([words[0],words[1],state_idx])
            else:
                new_line.append([words[0],words[1],last_tag+'_'+state_idx])
            
            if state_idx=='E':
                cur_idx = -2
                while True:
                    cur_wd = new_line[cur_idx]
                    if cur_wd[2]=='B' or cur_wd[2]=='I':
                        cur_wd[2]=last_tag+'_'+cur_wd[2]
                    else:
                        break
                    cur_idx=cur_idx-1
                    if -cur_idx>len(new_line):
                        break
                'E-->O'
                state_idx = 'O' 
                last_tag=''
            'B-->I'   
            if state_idx=='B':
                state_idx = 'I'
        neclear_corpor.append(new_line)
    
    spt_idx = int(len(neclear_corpor)*0.9)
    netrain_corpus = neclear_corpor[0:spt_idx]
    netest_corpus = neclear_corpor[spt_idx+1:]
    
    fd = codecs.open('people_daily_netrain.txt','w',encoding='gbk') 
    for ln in netrain_corpus:
        for wd in ln:
            fd.write(wd[0]+' '+wd[1]+' '+wd[2]+'\n')
        fd.write('\n')
    fd.close()
    
    fd = codecs.open('people_daily_netest.txt','w',encoding='gbk') 
    for ln in netest_corpus:
        for wd in ln:
            fd.write(wd[0]+' '+wd[1]+' '+wd[2]+'\n')
        fd.write('\n')
    fd.close()
        